const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  let tab = 'other';
  let sub_tab = '美团';
  let source = '美团技术团队';
  const fileName = `meituan.json`;
  let totalBlogs = [];
  let blogs = [];

  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});

  page.on('pageerror', msg => {
    console.log(msg);
  });
  page.on('console', msg => {
    console.log(msg.text());
  });

  for (let i = 1; i < 20; i++) {
    // if (i > 1) {
    //   break;
    // }
    let url = `https://tech.meituan.com/page/${i}.html`;
    await page.goto(url, {waitUntil: 'domcontentloaded'});

    await page.waitFor(1000);
    blogs = await page.$$eval('#J_main-container > div > div.col-md-6', async (eles) => {
      let tmpBlogs = [];
      eles.forEach(async ele => {
        let title = ele.querySelector('div > h2 > a').innerText;
        let content = ele.querySelector('div > h2 > a').href;
        let intro = ele.querySelector('div > div.post-content.post-expect').innerText;
        let create_at = ele.querySelector('div > div:nth-child(2) > span.m-post-date').innerText;
        create_at = create_at.replace(/(\d+)年(\d+)月(\d+)日/g, '$1-$2-$3');
        tmpBlogs.push({
          title: title,
          content: content,
          intro: intro,
          tab: 'other',
          sub_tab: '美团',
          source: '美团技术团队',
          create_at: create_at
        });
      });
      return tmpBlogs;
    });

    console.log(i, 20, blogs.length);
    totalBlogs = totalBlogs.concat(blogs);

  }

  let writerStream = fs.createWriteStream(`crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

};


main();


